Stable Diffusion

Note

Install ekorpkit package first.

Set logging level to Warning, if you don’t want to see verbose logging.

If you run this notebook in Colab, set Hardware accelerator to GPU.

!pip install ekorpkit[art] exit()

from ekorpkit import eKonf

eKonf.setLogger("INFO")
eKonf.set_cuda(device="4,5")
print("version:", eKonf.__version__)

is_notebook = eKonf.is_notebook()
is_colab = eKonf.is_colab()
print("is notebook?", is_notebook)
print("is colab?", is_colab)
if is_colab:
    eKonf.mount_google_drive(
        workspace="MyDrive/colab_workspace", project="disco-imagen"
    )

print("environment variables:")
eKonf.print(eKonf.env().dict())

INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB (id:4)', 'A100-SXM4-40GB (id:5)']
INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell
INFO:ekorpkit.utils.notebook:Google Colab not detected.

version: 0.1.40.post0.dev1
is notebook? True
is colab? False
environment variables:
{'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',
 'CUDA_VISIBLE_DEVICES': '4, 5',
 'EKORPKIT_CONFIG_DIR': '/workspace/projects/ekorpkit-book/config',
 'EKORPKIT_DATA_DIR': None,
 'EKORPKIT_LOG_LEVEL': 'INFO',
 'EKORPKIT_PROJECT': 'ekorpkit-book',
 'EKORPKIT_WORKSPACE_ROOT': '/workspace',
 'KMP_DUPLICATE_LIB_OK': 'TRUE',
 'NUM_WORKERS': 230}

Create a stable diffusion instance

cfg = eKonf.compose("model/stable_diffusion")
sd = eKonf.instantiate(cfg)

INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env
INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell
INFO:ekorpkit.base:setting environment variable CACHED_PATH_CACHE_ROOT to /workspace/.cache/cached_path
INFO:ekorpkit.base:setting environment variable KMP_DUPLICATE_LIB_OK to TRUE
INFO:absl:Unable to initialize backend 'tpu_driver': NOT_FOUND: Unable to find driver in registry given worker: 
INFO:absl:Unable to initialize backend 'rocm': NOT_FOUND: Could not find registered platform with name: "rocm". Available platform names are: Interpreter Host CUDA
INFO:absl:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'
INFO:ekorpkit.models.art.base:> downloading models...
INFO:ekorpkit.models.art.base:> loading modules...
INFO:ekorpkit.models.art.base:No modules to load
INFO:ekorpkit.models.art.base:> loading models...

Generate images

text_prompts = "a living tiger inside a gorgeous cave in a futuristic Jeju island"
batch_name = "cave-jeju"

sd.imagine(
    text_prompts, 
    batch_name=batch_name, 
    num_samples=2, 
    show_collage=True,
    guidance_scale=2
)

INFO:ekorpkit.models.art.base:Merging config with args: {}
INFO:ekorpkit.models.art.base:Prompt: None

INFO:ekorpkit.models.art.base:Saving config to /workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(1)_settings.yaml

{'image_filepaths': ['/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(1)_0000.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(1)_0001.png'],
 'config_file': 'cave-jeju(1)_settings.yaml',
 'config': {'batch_name': 'cave-jeju',
  'mode': 'generate',
  'text_prompts': {0: ['a living tiger inside a gorgeous cave in a futuristic Jeju island']},
  'batch_num': 1,
  'num_samples': 2,
  'num_images_per_prompt': 1,
  'width': 768,
  'height': 512,
  'num_inference_steps': 100,
  'guidance_scale': 2,
  'init_image': None,
  'mask_image': None,
  'inpaint_strength': 0.75,
  'set_seed': 'random_seed',
  'seed': 710849687,
  'resume_run': False,
  'run_to_resume': 'latest'}}

Inpaint images

base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
mask_image_path = f"{base_url}/chu-horse-masking-rough.png"
mask_image_exact_path = f"{base_url}/chu-horse-masking-exact.png"

init_image = eKonf.load_image(init_image_path)
mask_image = eKonf.load_image(mask_image_path)
mask_image_exact = eKonf.load_image(mask_image_exact_path)

sd.compare_images([mask_image, init_image, mask_image_exact], resize_ratio=0.5)

prompt = "a golden statue of a eagle"

images = sd.imagine(
    prompt, 
    mode="inpaint",
    init_image=init_image_path,
    mask_image=mask_image_path,
    num_samples=2,
    num_images_per_prompt=2, 
    guidance_scale=3, 
    num_inference_steps=50,
    return_including_init_image=False
)

INFO:ekorpkit.models.art.base:Merging config with args: {}
INFO:ekorpkit.models.art.base:Prompt: None

Stitch images

sd.imagine(
    text_prompts, 
    mode="stitch",
    batch_name=batch_name, 
    num_samples=6, 
    show_collage=True,
)

INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell

INFO:ekorpkit.models.art.stable: >> elapsed time to imagine: 0:01:39.323883
INFO:ekorpkit.models.art.base:Saving config to /workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_settings.yaml

{'stitched_image_path': '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_panorama.png',
 'image_filepaths': ['/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0000.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0001.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0002.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0003.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0004.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0005.png'],
 'config_file': 'cave-jeju(4)_settings.yaml',
 'config': {'batch_name': 'cave-jeju',
  'mode': 'Stitch',
  'text_prompts': {0: ['a living tiger inside a gorgeous cave in a futuristic Jeju island']},
  'batch_num': 4,
  'num_samples': 6,
  'num_images_per_prompt': 1,
  'width': 768,
  'height': 512,
  'num_inference_steps': 100,
  'guidance_scale': None,
  'init_image': None,
  'mask_image': None,
  'inpaint_strength': 0.75,
  'set_seed': 'random_seed',
  'seed': 501160171,
  'resume_run': False,
  'run_to_resume': 'latest'}}

To view the panorama, upload the image to renderstuff.

Show config

sd.show_config(batch_name=batch_name, batch_num=0)

INFO:ekorpkit.models.art.base:Loading config from /workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(0)_settings.yaml
INFO:ekorpkit.models.art.base:Merging config with diffuse defaults
INFO:ekorpkit.models.art.base:Merging config with args: {}

{'batch_name': 'cave-jeju',
 'batch_num': 3,
 'guidance_scale': None,
 'height': 512,
 'init_image': None,
 'inpaint_strength': 0.75,
 'mask_image': None,
 'mode': 'Stitch',
 'num_images_per_prompt': 1,
 'num_inference_steps': 100,
 'num_samples': 6,
 'resume_run': False,
 'run_to_resume': 'latest',
 'seed': 2034638016,
 'set_seed': 'random_seed',
 'text_prompts': {0: ['a living tiger inside a gorgeous cave in a futuristic '
                      'Jeju island']},
 'width': 768}